ICLR Paper - Data Analysis

Import Dependencies & Data

In [1]:
#%matplotlib notebook
import os
import matplotlib.pyplot as plt
import numpy as np
from JSAnimation import IPython_display
from matplotlib import animation
import matplotlib.patches as mpatches
import cv2
import imageio
import mpld3
import scipy.misc
from RL_utils import *
import tensorflow as tf
import tensorflow.contrib.layers as c_layers
import seaborn as sns
import pandas as pd
import glob
from scipy import misc
In [2]:
folder_name = '3999_16.100'
path = './Results/TowerTraining/'+folder_name+'/'

obs = np.load(path+'visobs.npy', mmap_mode='r')
vec = np.load(path+'vecobs.npy')
enc = np.load(path+'encodings.npy')
val = np.load(path+'values.npy')
act = np.load(path+'actions.npy')
rew = np.load(path+'rewards.npy')

doors = np.load(path+'doors.npy')
Rewards = np.load(path+'reward_per.npy')

hand_l = pd.read_csv(path+'HandLabels.csv')
label_test = np.zeros(enc.shape[0])
label_test = np.array(hand_l['Label'])

Get Random Agent Activations

In [3]:
tf.reset_default_graph()

def swish(input_activation):
    """Swish activation function. For more info: https://arxiv.org/abs/1710.05941"""
    return tf.multiply(input_activation, tf.nn.sigmoid(input_activation))

def create_global_steps():
    """Creates TF ops to track and increment global training step."""
    global_step = tf.Variable(0, name="global_step", trainable=False, dtype=tf.int32)
    increment_step = tf.assign(global_step, tf.add(global_step, 1))
    return global_step, increment_step


global_step, increment_step = create_global_steps()

o_size_h = 168
o_size_w = 168
vec_obs_size = 8
num_layers = 2
h_size = 256
h_size_vec = 256
            
visual_in = tf.placeholder(shape=[None, o_size_h, o_size_w, 3], dtype=tf.float32,name="visual_observation_0")

vector_in = tf.placeholder(shape=[None, vec_obs_size], dtype=tf.float32,name="vector_observation_0")

running_mean = tf.get_variable("running_mean", [vec_obs_size],trainable=False, dtype=tf.float32,initializer=tf.zeros_initializer())
running_variance = tf.get_variable("running_variance", [vec_obs_size],trainable=False,dtype=tf.float32,initializer=tf.ones_initializer())

mean_current_observation = tf.reduce_mean(vector_in, axis=0)
new_mean = running_mean + (mean_current_observation - running_mean) / tf.cast(tf.add(global_step, 1), tf.float32)
new_variance = running_variance + (mean_current_observation - new_mean) * (mean_current_observation - running_mean)
update_mean = tf.assign(running_mean, new_mean)
update_variance = tf.assign(running_variance, new_variance)

normalized_state = tf.clip_by_value((vector_in - running_mean) / tf.sqrt(running_variance / (tf.cast(global_step, tf.float32) + 1)), -5, 5,name="normalized_state")

def create_vector_observation_encoder(observation_input, h_size, activation, num_layers, scope,reuse):
    with tf.variable_scope(scope):
        hidden_vec = observation_input
        for i in range(num_layers):
            hidden_vec = tf.layers.dense(hidden_vec, h_size, activation=activation, reuse=reuse,name="hidden_{}".format(i),kernel_initializer=c_layers.variance_scaling_initializer(1.0))
    return hidden_vec

def create_visual_observation_encoder(image_input, h_size, activation, num_layers, scope,reuse):
    with tf.variable_scope(scope):
        conv1 = tf.layers.conv2d(image_input, 16, kernel_size=[8, 8], strides=[4, 4],activation=tf.nn.elu, reuse=reuse, name="conv_1")
        conv2 = tf.layers.conv2d(conv1, 32, kernel_size=[4, 4], strides=[2, 2],activation=tf.nn.elu, reuse=reuse, name="conv_2")
        hidden_vis = c_layers.flatten(conv2)

    with tf.variable_scope(scope + '/' + 'flat_encoding'):
        hidden_flat = create_vector_observation_encoder(hidden_vis, h_size, activation,num_layers, scope, reuse)
    return hidden_flat


visual_encoders = []
hidden_state, hidden_visual = None, None

encoded_visual = create_visual_observation_encoder(visual_in,h_size,swish,num_layers,"main_graph_0_encoder0", False)
visual_encoders.append(encoded_visual)
hidden_visual = tf.concat(visual_encoders, axis=1)

hidden_state = create_vector_observation_encoder(vector_in,h_size_vec, swish,num_layers,"main_graph_0",False)

final_hidden = tf.concat([hidden_visual, hidden_state], axis=1)

sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)

random_enc = sess.run(final_hidden, feed_dict={visual_in: obs, vector_in: vec[0]})

Data Preparation

In [4]:
encodings = np.zeros((obs.shape[0],16,32))
for i in range(obs.shape[0]):
    encodings[i] = enc[i].reshape((16,32)) 
enc2D = enc.reshape(enc.shape[0],enc.shape[-1])
print(enc2D.shape)

R_encodings = np.zeros((obs.shape[0],16,32))
for i in range(obs.shape[0]):
    R_encodings[i] = random_enc[i].reshape((16,32)) 
R_enc2D = random_enc.reshape(random_enc.shape[0],random_enc.shape[-1])
print(R_enc2D.shape)
(4000, 512)
(4000, 512)

Run Statistics

In [65]:
fig = plt.figure(figsize=(30,10))
ax = plt.subplot(1,2,1)
plt.plot(vec[0][:,7],linewidth=7)
plt.plot(rew,linewidth=5)
plt.plot(val[:,:,0],linewidth=5)
plt.legend(['Level','Reward','Value Estimate'], fontsize=30)
plt.title('Inference Run - Agent Performance', fontsize=40)
plt.xlabel('Steps',fontsize=30)
plt.rc('xtick',labelsize=20)
plt.rc('ytick',labelsize=20)
plt.subplot(1,2,2)
plt.title('Agent Observations\n',fontsize=40)
image = misc.imread(path+'figures/ExampleObs.jpg')
plt.imshow(image)
plt.axis('off')

#plt.show()
#mpld3.save_html(fig, 'myfig.html')
plt.savefig(path+'figures/TrainStatistics.eps', bbox_inches='tight', format='eps')
plt.savefig(path+'figures/TrainStatistics.png', bbox_inches='tight')
c:\users\vkakerbeck\miniconda3\lib\site-packages\ipykernel_launcher.py:13: DeprecationWarning: `imread` is deprecated!
`imread` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imread`` instead.
  del sys.path[0]
In [6]:
fig = plt.figure(figsize=(15,10))
#ax = plt.subplot(1,2,1)
plt.plot(vec[0][:,7],linewidth=8)
plt.plot(val[:,:,0],linewidth=5)
plt.plot(rew,linewidth=3)
plt.legend(['Level','Value Estimate','Reward'], fontsize=30)
plt.title('Inference Run - Agent Performance', fontsize=40)
plt.xlabel('Steps',fontsize=30)
plt.rc('xtick',labelsize=20)
plt.rc('ytick',labelsize=20)
plt.xlim(0,4000)
plt.ylim(0,11)

plt.savefig(path+'figures/TrainStatistics2.eps', bbox_inches='tight', format='eps')
plt.savefig(path+'figures/TrainStatistics2.png', bbox_inches='tight')
#plt.show()
In [7]:
print('Reward is received in '+str(rew[rew>0].shape[0])+' out of '+str(rew.shape[0])+' frames ('+str(rew[rew>0].shape[0]/rew.shape[0]*100)+'%).')
print('Only '+str(rew[rew>0.9].shape[0])+' of these contain a full reward of 1.')
Reward is received in 76 out of 4000 frames (1.9%).
Only 15 of these contain a full reward of 1.
In [251]:
f = plot_actions(act)
#plt.show()
plt.savefig(path+'figures/ActionDistributions.eps', bbox_inches='tight')

Get Action Combination at Each Frame

In [22]:
def where_array_equal(a,b):
    equal = np.zeros(a.shape[0])
    for i,arr in enumerate(a):
        if np.array_equal(arr,b):
            equal[i] = 1
    return equal
w = where_array_equal(act[:,0],np.unique(act[:,0],axis=0)[3])
In [23]:
combinations = np.unique(act[:,0],axis=0)
a_vec = np.zeros((combinations.shape[0],act.shape[0]))
for i,c in enumerate(combinations):
    a_vec[i] = where_array_equal(act[:,0],c)
print(a_vec.shape)    
    
(16, 4000)

Merge Combinations

In [24]:
all_a_comb = np.zeros(act.shape[0])
for i in range(act.shape[0]):
    if a_vec[0][i] == 1:
        all_a_comb[i] = 0#[1, 0, 0, 1] Forward, Turn right
    elif a_vec[3][i] == 1:
        all_a_comb[i] = 1#[1, 1, 0, 1] Forward, Camera Left, Turn Right
    elif a_vec[4][i] == 1:
        all_a_comb[i] = 2#[1, 1, 0, 2] Forward, Camera Left, Turn Left
    elif a_vec[6][i] == 1:
        all_a_comb[i] = 3#[1, 2, 0, 1] Forward, Camera Right, Turn Right
    elif a_vec[13][i] == 1:
        all_a_comb[i] = 4#[2, 2, 0, 1] Backward, Camera Right, Turn Right
    else:
        all_a_comb[i] = 5#Anything else
In [107]:
action_C = ['Forward,\nTurn Right','Forward,\nCamera Left,\nTurn Right','Forward,\nCamera Left,\nTurn Left',
           'Forward,\nCamera Right,\nTurn Right','Backward,\nCamera Right,\nTurn Right','Everything\nElse']

plt.figure(figsize=(10,15))
plt.subplots_adjust(hspace=0.2,wspace=0.01)
ax1 = plt.subplot(2,1,1)
plt.title("Distribution of Frames for all Action Combinations",fontsize=20)

p1 = np.array(sns.color_palette("Accent", n_colors=8))[np.array([0,7,7,1,3,7,4,7,7,7,7,7,7,6,7])]

b1 = sns.barplot(x=np.linspace(0,15,16),y=np.sum(a_vec,axis=1),palette=p1)

plt.xlabel("Action Combination",fontsize=15)
plt.ylabel("# of Frames",fontsize=15)
plt.xticks(np.linspace(0,15,16),np.linspace(0,15,16,dtype=int))
#plt.yscale('log')

ax2 = plt.subplot(2,1,2)
plt.title("Distribution of Frames for Selected Action Combinations",fontsize=20)

p2 = np.array(sns.color_palette("Accent", n_colors=8))[np.array([0,1,3,4,6,7])]
a2 = np.histogram(all_a_comb,bins=np.linspace(-0.5,5.5,7))
b2 = sns.barplot(x=np.linspace(0,5,6),y=a2[0],palette=p2,ax=ax2)

plt.xlabel("Action Combination",fontsize=15)
plt.ylabel("# of Frames",fontsize=15)
plt.xticks(np.linspace(0,5,6),action_C)
#plt.show()
plt.savefig(path+'figures/ActionCombDist.eps', bbox_inches='tight')

Agent State Representation

In [10]:
end = 200
plot_movie_jsInfo(encodings[:end],obs[:end],act[:end],val[:end],rew[:end])#,save=path+'encodingsInfo.mp4')


Once Loop Reflect
In [11]:
%%capture
end=200
save_movie_jsInfo(encodings[:end],obs[:end],act[:end],val[:end],rew[:end],save=path+'frames/encoding')
In [62]:
perAct = np.sum((enc2D)>0,axis=0)/enc2D.shape[0]
fig = plt.figure()
ax1 = fig.add_subplot(2, 1, 1)
im1 = plt.imshow(perAct[:256].reshape(8,32), vmin=0, vmax=1)
plt.title('Visual Encoding',fontsize=15)
plt.axis('off')
ax2 = fig.add_subplot(2, 1, 2)
im2 = plt.imshow(perAct[256:].reshape(8,32), vmin=0, vmax=1)
plt.title('Vector Encoding',fontsize=15)
plt.axis('off')
plt.suptitle('Trained Agent',fontsize=18)
fig.subplots_adjust(right=0.8,top=0.85,bottom=0.2)
cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.75])
cbar = fig.colorbar(im1, cax=cbar_ax)
cbar.ax.set_yticklabels(['0%', '20%', '40%','60%','80%','100%'],fontsize=12)
cbar.set_label('Active in % of Frames', rotation=270,fontsize=15, labelpad=10)

plt.show()
#plt.savefig(path+'figures/PerAct.eps', bbox_inches='tight')
In [65]:
print("Trained Agent:")
print("Average number of neurons active per frame: "+str(np.mean(np.sum(enc2D>0,axis=1)))+" (min="+str(np.min(np.sum(enc2D>0,axis=1)))+
      " max="+str(np.max(np.sum(enc2D>0,axis=1)))+" var="+str(np.round(np.var(np.sum(enc2D>0,axis=1)),3))+")")
print("This is "+str(np.round(np.mean(np.sum(enc2D>0,axis=1))/512*100,3))+"% of all neurons.")
print("Average number of neurons active per frame (visual): "+str(np.mean(np.sum(enc2D[:,:256]>0,axis=1)))+" (min="+str(np.min(np.sum(enc2D[:,:256]>0,axis=1)))+
      " max="+str(np.max(np.sum(enc2D[:,:256]>0,axis=1)))+" var="+str(np.round(np.var(np.sum(enc2D[:,:256]>0,axis=1)),3))+")")
print("This is "+str(np.round(np.mean(np.sum(enc2D[:,:256]>0,axis=1))/256*100,3))+"% of all neurons.")
visActPer = np.sum((enc2D[:,:256])>0,axis=0)/enc2D.shape[0]*100
print("In the visual encoding "+str(np.sum(visActPer>0))+" of the 256 neurons are active in at least 1 frame (="+
     str(np.round(np.sum(visActPer>0)/256*100,3))+"%), "+str(np.sum(visActPer>40))+" neurons are active in more than 40% of the frames (="+
     str(np.round(np.sum(visActPer>40)/256*100,3))+"%). "+str(np.sum(visActPer==0))+" neurons are never active (="+str(np.round(np.sum(visActPer==0)/256*100,3))+
     "%). The most active neuron is active in "+str(np.max(visActPer))+"% of the frames.")
vecActPer = np.sum((enc2D[:,256:])>0,axis=0)/enc2D.shape[0]*100

print("In the vector encoding "+str(np.sum(vecActPer>0))+" of the 256 neurons are active in at least 1 frame (="+
     str(np.round(np.sum(vecActPer>0)/256*100,3))+"%), "+str(np.sum(vecActPer>40))+" neurons are active in more than 40% of the frames (="+
     str(np.round(np.sum(vecActPer>40)/256*100,3))+"%). "+str(np.sum(vecActPer==0))+" neurons are never active (="+str(np.round(np.sum(vecActPer==0)/256*100,3))+
      "%). The most active neuron is active in "+str(np.max(vecActPer))+"% of the frames.")
Trained Agent:
Average number of neurons active per frame: 15.76 (min=6 max=39 var=20.795)
This is 3.078% of all neurons.
Average number of neurons active per frame (visual): 11.8805 (min=3 max=35 var=17.891)
This is 4.641% of all neurons.
In the visual encoding 173 of the 256 neurons are active in at least 1 frame (=67.578%), 7 neurons are active in more than 40% of the frames (=2.734%). 83 neurons are never active (=32.422%). The most active neuron is active in 74.15% of the frames.
In the vector encoding 25 of the 256 neurons are active in at least 1 frame (=9.766%), 3 neurons are active in more than 40% of the frames (=1.172%). 231 neurons are never active (=90.234%). The most active neuron is active in 100.0% of the frames.
In [64]:
print("Average number of neurons active per frame (visual): "+str(np.mean(np.sum(enc2D[:,:256]>0,axis=1)))+" (min="+str(np.min(np.sum(enc2D[:,:256]>0,axis=1)))+
      " max="+str(np.max(np.sum(enc2D[:,:256]>0,axis=1)))+" var="+str(np.round(np.var(np.sum(enc2D[:,:256]>0,axis=1)),3))+")")
Average number of neurons active per frame (visual): 11.8805 (min=3 max=35 var=17.891)
In [296]:
perAct = np.sum((R_enc2D)>0,axis=0)/enc2D.shape[0]
fig = plt.figure()
ax1 = fig.add_subplot(2, 1, 1)
im1 = plt.imshow(perAct[:256].reshape(8,32), vmin=0, vmax=1)
plt.title('Visual Encoding',fontsize=15)
plt.axis('off')
ax2 = fig.add_subplot(2, 1, 2)
im2 = plt.imshow(perAct[256:].reshape(8,32), vmin=0, vmax=1)
plt.title('Vector Encoding',fontsize=15)
plt.axis('off')
plt.suptitle('Random Agent',fontsize=18)
fig.subplots_adjust(right=0.8,top=0.85,bottom=0.2)
cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.75])
cbar = fig.colorbar(im1, cax=cbar_ax)
cbar.ax.set_yticklabels(['0%', '20%', '40%','60%','80%','100%'],fontsize=12)
cbar.set_label('Active in % of Frames', rotation=270,fontsize=15, labelpad=10)

plt.show()
#plt.savefig(path+'figures/PerAct_R.eps', bbox_inches='tight')
In [298]:
print("Random Agent:")
print("Average number of neurons active per frame: "+str(np.mean(np.sum(R_enc2D>0,axis=1)))+" (min="+str(np.min(np.sum(R_enc2D>0,axis=1)))+
      " max="+str(np.max(np.sum(R_enc2D>0,axis=1)))+" var="+str(np.round(np.var(np.sum(R_enc2D>0,axis=1)),3))+")")
print("This is "+str(np.round(np.mean(np.sum(R_enc2D>0,axis=1))/512*100,3))+"% of all neurons.")
visActPer = np.sum((R_enc2D[:,:256])>0,axis=0)/R_enc2D.shape[0]*100
print("In the visual encoding "+str(np.sum(visActPer>0))+" of the 256 neurons are active in at least 1 frame (="+
     str(np.round(np.sum(visActPer>0)/256*100,3))+"%), "+str(np.sum(visActPer>40))+" neurons are active in more than 40% of the frames (="+
     str(np.round(np.sum(visActPer>40)/256*100,3))+"%). "+str(np.sum(visActPer==0))+" neurons are never active (="+str(np.round(np.sum(visActPer==0)/256*100,3))+
     "%). The most active neuron is active in "+str(np.max(visActPer))+"% of the frames.")
vecActPer = np.sum((R_enc2D[:,256:])>0,axis=0)/R_enc2D.shape[0]*100

print("In the vector encoding "+str(np.sum(vecActPer>0))+" of the 256 neurons are active in at least 1 frame (="+
     str(np.round(np.sum(vecActPer>0)/256*100,3))+"%), "+str(np.sum(vecActPer>40))+" neurons are active in more than 40% of the frames (="+
     str(np.round(np.sum(vecActPer>40)/256*100,3))+"%). "+str(np.sum(vecActPer==0))+" neurons are never active (="+str(np.round(np.sum(vecActPer==0)/256*100,3))+
      "%). The most active neuron is active in "+str(np.max(vecActPer))+"% of the frames.")
Random Agent:
Average number of neurons active per frame: 253.1915 (min=227 max=277 var=34.5)
This is 49.451% of all neurons.
In the visual encoding 254 of the 256 neurons are active in at least 1 frame (=99.219%), 152 neurons are active in more than 40% of the frames (=59.375%). 2 neurons are never active (=0.781%). The most active neuron is active in 100.0% of the frames.
In the vector encoding 125 of the 256 neurons are active in at least 1 frame (=48.828%), 125 neurons are active in more than 40% of the frames (=48.828%). 131 neurons are never active (=51.172%). The most active neuron is active in 100.0% of the frames.

PCA

In [333]:
from sklearn.decomposition import PCA
from mpl_toolkits.mplot3d import Axes3D
In [15]:
num_comp = 20
pca = PCA(n_components=num_comp)
pca.fit(enc2D[:,:256])#only for visual embedding
pcs = pca.transform(enc2D[:,:256])

r_pca = PCA(n_components=num_comp)
r_pca.fit(R_enc2D[:,:256])#only for visual embedding
r_pcs = r_pca.transform(R_enc2D[:,:256])
In [24]:
fig = plt.figure(figsize=(15,5))

ax1 = fig.add_subplot(1, 2, 1, projection='3d')
ax1.scatter(pcs[:,0],pcs[:,1],pcs[:,2])
plt.title('Trained Agent')

ax2 = fig.add_subplot(1, 2, 2, projection='3d')
ax2.scatter(r_pcs[:,0],r_pcs[:,1],r_pcs[:,2])
plt.title('Random Agent')

plt.suptitle('Visual Embeddings Projected on 3 PCs',fontsize = 15)
plt.show()
#plt.savefig(path+'figures/PCA3_Vis.eps', bbox_inches='tight')
In [21]:
plt.figure(figsize=(15,5))

plt.subplot(1,2,1)
plt.bar(np.linspace(1,20,20),pca.explained_variance_ratio_)
plt.ylim((0,0.3))
plt.title('Trained Agent')

plt.subplot(1,2,2)
plt.bar(np.linspace(1,20,20),r_pca.explained_variance_ratio_)
plt.ylim((0,0.3))
plt.title('Random Agent')

plt.suptitle('Ration of Explained Variance of First 20 PCs')
plt.show()
#plt.savefig(path+'figures/PCA'+str(num_comp)+'_ExpVar.png', bbox_inches='tight')
In [22]:
print('Trained Agent: ' + str(pca.explained_variance_ratio_.cumsum()))
print('Random Agent: ' + str(r_pca.explained_variance_ratio_.cumsum()))
Trained Agent: [0.27886975 0.43800944 0.5852343  0.6264317  0.661513   0.6908839
 0.71823967 0.74123347 0.7627876  0.78255236 0.798869   0.8138639
 0.82775015 0.84009683 0.85105723 0.8618459  0.87083924 0.8785413
 0.885939   0.8929034 ]
Random Agent: [0.12627248 0.20060867 0.25757185 0.30136392 0.33326244 0.3624758
 0.38772732 0.41018227 0.4306385  0.45011795 0.4682609  0.48490173
 0.49981877 0.5136789  0.5265416  0.53917986 0.55053824 0.561466
 0.5721834  0.5827901 ]

Meaningful Cluster - Correlations with Actions

K-Means Clustering

In [11]:
from sklearn.cluster import KMeans

Best Number of Cluster?

In [12]:
def getClusterVariance(cluster,data,num_cluster):
    between_var = np.var(data,axis=0)
    within_var = []
    for i in range(num_cluster):
        which = np.where(cluster==i)
        within_var.append(np.var(data[which],axis=0))
    return between_var, within_var
In [13]:
wss = []
med_wss = []
for num_cluster in range(2,11):
    kmeans = KMeans(n_clusters=num_cluster, random_state=0).fit(enc2D[:,:256])
    cluster = kmeans.predict(enc2D[:,:256])
    _,w = getClusterVariance(cluster,enc2D[:,:256],num_cluster)
    wss.append(np.mean(np.array(w)))
    med_wss.append(np.median(np.array(w)))
In [190]:
plt.figure()
plt.plot(np.array(wss),color='green')
plt.xticks(np.linspace(0,8,9),np.linspace(2,10,9,dtype=int))
plt.title('Mean Within Class Variance Comparison',fontsize=15)
plt.xlabel('# of Cluster',fontsize=15)
plt.ylabel('Mean Within Class Variance',fontsize=15)
#plt.show()
plt.savefig(path+'figures/WithinVarComparisonMean.eps', bbox_inches='tight')
In [162]:
from sklearn.metrics import silhouette_score
scs = []
for num_cluster in range(2,11):
    kmeans = KMeans(n_clusters=num_cluster, random_state=0).fit(enc2D[:,:256])
    cluster = kmeans.predict(enc2D[:,:256])
    sc = silhouette_score(enc2D[:,:256], cluster, metric='euclidean')
    scs.append(np.array(sc))
In [176]:
plt.figure()
plt.plot(np.array(scs),color='green')
plt.title('Silhouette Score Comparison',fontsize=15)
plt.xlabel('# of Cluster',fontsize=15)
plt.ylabel('Silhouette Score',fontsize=15)
plt.xticks(np.linspace(0,8,9),np.linspace(2,10,9,dtype=int))
#plt.show()
plt.savefig(path+'figures/SilhouetteScoreComp.eps', bbox_inches='tight')

Final Clustering

In [14]:
num_cluster = 6
kmeans = KMeans(n_clusters=num_cluster, random_state=0).fit(enc2D[:,:256])
cluster = kmeans.predict(enc2D[:,:256])

r_kmeans = KMeans(n_clusters=num_cluster, random_state=0).fit(R_enc2D[:,:256])
r_cluster = r_kmeans.predict(R_enc2D[:,:256])
In [16]:
ba,wa = getClusterVariance(cluster,enc2D[:,:256],num_cluster)
b = np.mean(ba)
w = np.zeros(num_cluster)
for i in range(num_cluster):
    w[i] = np.mean(wa[i])
plt.figure()
plt.title('Within Class Variance',fontsize=20)
plt.bar(np.linspace(0,num_cluster-1,num_cluster),np.array(w),color=[(0.49, 0.5+c*0.09, 0.3 + c*0.05)for c in range(6)])
plt.plot([-1,num_cluster],[np.mean(np.array(w)),np.mean(np.array(w))],color='green')
plt.plot([-1,num_cluster],[b,b],color='black')
plt.xlim([-0.5,num_cluster-0.5])
plt.legend(['Avg Within Class Variance','Overall Variance'],fontsize=10)
plt.xlabel('Cluster',fontsize=15)
plt.ylabel('Variance',fontsize=15)
#plt.show()
#plt.savefig(path+'figures/WithinVar6.eps', bbox_inches='tight')
plt.savefig(path+'figures/WithinVar6.png', bbox_inches='tight')
In [17]:
plt.figure()
h = np.histogram(cluster,bins=np.linspace(-0.5,num_cluster-0.5,num_cluster+1))
plt.bar(np.linspace(0,5,6),h[0],color=[(0.49, 0.5+c*0.09, 0.3 + c*0.05)for c in range(6)])
plt.title('Distribution of Frames in Cluster', fontsize = 20)
plt.xlabel('Cluster', fontsize = 15)
plt.ylabel('# of Frames', fontsize = 15)
#plt.show()
#plt.savefig(path+'figures/KMeans_'+str(num_cluster)+'_Distribution.eps', bbox_inches='tight')
plt.savefig(path+'figures/KMeans_'+str(num_cluster)+'_Distribution.png', bbox_inches='tight')
In [18]:
%%capture
lens = []
for i in range(num_cluster):
    which = np.where(cluster==i)
    #print(cluster[which].shape)
    lens.append(cluster[which].shape[0])
    os.mkdir(path+'frames/cluster_frames_'+str(num_cluster)+'/C'+str(i))
    save_movie_js2(encodings[which],obs[which],save=path+'frames/cluster_frames_'+str(num_cluster)+'/C'+str(i))
In [19]:
print(lens) #Add this into html file
[1062, 1004, 252, 463, 1027, 192]

Meaningfulness of Cluster

In [25]:
p = np.array(sns.color_palette("Accent", n_colors=8))[np.array([0,1,3,4,6,7])]
#%%capture
#actions = ['Forward/Back','Camera','Jump','Left/Right']
actions = ['Forward, Turn Right','Forward, Camera Left, Turn Right','Forward, Camera Left, Turn Left',
           'Forward, Camera Right, Turn Right','Backward, Camera Right, Turn Right','Everything Else']
corVal=np.zeros((6,num_cluster))
for action in range(6):
    corrs = np.zeros((21,num_cluster))
    for c in range(num_cluster):
        for r in range(-10,11):
            a = np.roll(all_a_comb==action, r)
            corrs[r+10,c] = np.corrcoef((cluster==c).astype(int),a)[1][0]
        corVal[action,c] = np.mean(corrs[5:15,c])
    plt.figure(figsize=(10,7))
    for c in range(num_cluster):
        #ysmoothed = gaussian_filter1d(corrs[:,c], sigma=0.5)
        plt.plot(corrs[:,c],linewidth=3,color=(0.49, 0.5+c*0.09, 0.3 + c*0.05))
    plt.legend(np.linspace(0,num_cluster-1,num_cluster,dtype=int),title='Cluster',title_fontsize=15,fontsize=13,loc=2,ncol=2)
    plt.title('Cluster Correlation with Action '+str(action)+' ('+actions[action]+')',fontsize=20)
    plt.xlabel('Sliding Window Offset',fontsize=15)
    plt.ylabel('Correlation Coefficient',fontsize=15)
    plt.xticks([0,5,10,15,20],['-10','-5','0','5','10'],fontsize=13)
    plt.ylim([-1,1])
    plt.xlim([0,20])
    #plt.show()
    plt.savefig(path+'frames/cluster_frames_'+str(num_cluster)+'/Corrcoef20_A'+str(action)+'.png', bbox_inches='tight')
In [26]:
df = pd.DataFrame(columns=['Action','cluster','correlation'])
for a in range(6):
    for c in range(num_cluster):
        df = df.append({'Action':actions[a],'cluster':int(c),'correlation':corVal[a,c]},ignore_index=True)
In [27]:
plt.figure(figsize=(10,7))
sns.barplot(x="cluster", y="correlation", hue="Action", data=df,palette=p)
plt.title('Avg. Correlation in 10 Frame Window Around Action with Cluster\nTrained Agent',fontsize=20)
plt.legend(fontsize=15,bbox_to_anchor=(1, 0.8), loc=2)
plt.ylabel('Correlation',fontsize=15)
plt.yticks(fontsize=15)
plt.ylim(-0.2,0.35)
plt.xlabel("Cluster",fontsize=15)
plt.xticks(fontsize=15)
#plt.show()
plt.savefig(path+'figures/Corrcoef_Comparison.png', bbox_inches='tight')

Randon Agent

In [28]:
#%%capture
#actions = ['Forward/Back','Camera','Jump','Left/Right']
actions = ['Forward, Turn Right','Forward, Camera Left, Turn Right','Forward, Camera Left, Turn Left',
           'Forward, Camera Right, Turn Right','Backward, Camera Right, Turn Right','Everything Else']
r_corVal=np.zeros((6,num_cluster))
for action in range(6):
    r_corrs = np.zeros((21,num_cluster))
    for c in range(num_cluster):
        for r in range(-10,11):
            a = np.roll(all_a_comb==action, r)
            r_corrs[r+10,c] = np.corrcoef((r_cluster==c).astype(int),a)[1][0]
        r_corVal[action,c] = np.mean(r_corrs[5:15,c])
    plt.figure(figsize=(10,7))
    for c in range(num_cluster):
        #ysmoothed = gaussian_filter1d(corrs[:,c], sigma=0.5)
        plt.plot(r_corrs[:,c],linewidth=3,color=(0.49, 0.5+c*0.09, 0.3 + c*0.05))
    plt.legend(np.linspace(0,num_cluster-1,num_cluster,dtype=int),title='Cluster',title_fontsize=15,fontsize=13,loc=2,ncol=2)
    plt.title('Cluster Correlation with Action '+str(action)+' ('+actions[action]+')',fontsize=20)
    plt.xlabel('Sliding Window Offset',fontsize=15)
    plt.ylabel('Correlation Coefficient',fontsize=15)
    plt.xticks([0,5,10,15,20],['-10','-5','0','5','10'],fontsize=13)
    plt.ylim([-1,1])
    plt.xlim([0,20])
    plt.show()
    #plt.savefig(path+'frames/cluster_frames_'+str(num_cluster)+'/Corrcoef20NoSmooth_r_A'+str(action)+'.png', bbox_inches='tight')
In [29]:
r_df = pd.DataFrame(columns=['Action','cluster','correlation'])
for a in range(6):
    for c in range(num_cluster):
        r_df = r_df.append({'Action':actions[a],'cluster':int(c),'correlation':r_corVal[a,c]},ignore_index=True)
In [30]:
plt.figure(figsize=(10,7))
sns.barplot(x="cluster", y="correlation", hue="Action", data=r_df,palette=p)
plt.title('Avg. Correlation in 10 Frame Window Around Action with Cluster \n Random Agent',fontsize=20)
plt.legend(fontsize=15,bbox_to_anchor=(1, 0.8), loc=2)
plt.ylabel('Correlation',fontsize=15)
plt.yticks(fontsize=15)
plt.ylim(-0.2,0.35)
plt.xlabel("Cluster",fontsize=15)
plt.xticks(fontsize=15)
plt.show()
#plt.savefig(path+'figures/Corrcoef_Comparison_R.eps', bbox_inches='tight')
In [102]:
action = 0
fig = plt.figure(figsize=(15,5))
plt.subplots_adjust(top=0.78,wspace=0.01)
plt.subplot(1,2,1)
corrs = np.zeros((21,num_cluster))
for c in range(num_cluster):
    for r in range(-10,11):
        a = np.roll(all_a_comb==action, r)
        corrs[r+10,c] = np.corrcoef((cluster==c).astype(int),a)[1][0]
for c in range(num_cluster):
    plt.plot(corrs[:,c],linewidth=3,color=(0.49, 0.5+c*0.09, 0.3 + c*0.05))
#plt.legend(np.linspace(0,num_cluster-1,num_cluster,dtype=int),title='Cluster',title_fontsize=15,fontsize=13,loc=2,ncol=2)
plt.title('Trained Agent',fontsize=18)
plt.xlabel('Sliding Window Offset',fontsize=15)
plt.ylabel('Correlation Coefficient',fontsize=15)
plt.xticks([0,5,10,15,20],['-10','-5','0','5','10    '],fontsize=13)
plt.ylim([-1,1])
plt.xlim([0,20])

plt.subplot(1,2,2)
r_corrs = np.zeros((21,num_cluster))
for c in range(num_cluster):
    for r in range(-10,11):
        a = np.roll(all_a_comb==action, r)
        r_corrs[r+10,c] = np.corrcoef((r_cluster==c).astype(int),a)[1][0]
for c in range(num_cluster):
    plt.plot(r_corrs[:,c],linewidth=3,color=(0.49, 0.5+c*0.09, 0.3 + c*0.05))
plt.legend(np.linspace(0,num_cluster-1,num_cluster,dtype=int),title='Cluster',title_fontsize=15,fontsize=13,loc=2,ncol=2)
plt.title('Random Agent',fontsize=18)
plt.xlabel('Sliding Window Offset',fontsize=15)
#plt.ylabel('Correlation Coefficient',fontsize=15)
plt.xticks([0,5,10,15,20],['    -10','-5','0','5','10'],fontsize=13)
plt.yticks([],[])
plt.ylim([-1,1])
plt.xlim([0,20])

plt.suptitle('Correlation in 20 Frame Window Around Action for \nAction = Forward + Turn Right',fontsize=20)
#plt.legend(fontsize=15,bbox_to_anchor=(1.15, 0.8), loc=2)
plt.show()
#plt.savefig(path+'figures/Corrcoef_Comparison_Expl.eps', bbox_inches='tight')
In [113]:
print("Trained Agent:\nMaximum Correlation: "+str(np.round(np.max(corrs),3))+" Minimum Correlation: "+str(np.round(np.min(corrs),3)))
print("Random Agent:\nMaximum Correlation: "+str(np.round(np.max(r_corrs),3))+" Minimum Correlation: "+str(np.round(np.min(r_corrs),3)))
Trained Agent:
Maximum Correlation: 0.577 Minimum Correlation: -0.253
Random Agent:
Maximum Correlation: 0.055 Minimum Correlation: -0.07
In [111]:
p = np.array(sns.color_palette("Accent", n_colors=8))[np.array([0,1,3,4,6,7])]

fig = plt.figure(figsize=(15,5))
plt.subplots_adjust(top=0.85,wspace=0.01)
plt.subplot(1,2,1)
ax = sns.barplot(x="cluster", y="correlation", hue="Action", data=df,palette=p)
ax.get_legend().remove()
plt.title('Trained Agent',fontsize=20)
plt.ylabel('Correlation',fontsize=15)
plt.yticks(fontsize=15)
plt.ylim(-0.18,0.45)
plt.xlabel("Cluster",fontsize=15)
plt.xticks(fontsize=15)

plt.subplot(1,2,2)
ax2 = sns.barplot(x="cluster", y="correlation", hue="Action", data=r_df,palette=p)
ax2.legend(fontsize=12,frameon=False,loc=2,title='Action Combination',title_fontsize=14)
plt.title('Random Agent',fontsize=20)
plt.ylabel('Correlation',fontsize=15)
plt.yticks(fontsize=15)
plt.ylim(-0.18,0.45)
ax2.yaxis.set_label_position("right")
ax2.yaxis.tick_right()
plt.yticks(fontsize=15)
plt.xlabel("Cluster",fontsize=15)
plt.xticks(fontsize=15)

plt.suptitle('Avg. Correlation in 10 Frame Window Around Action with Cluster',fontsize=22)
#plt.legend(fontsize=15,bbox_to_anchor=(1.15, 0.8), loc=2)
#plt.show()
plt.savefig(path+'figures/Corrcoef_Comparison_both.eps', bbox_inches='tight')
In [104]:
sos = np.zeros(num_cluster)#sum of squares of correlations in one cluster
for c in range(num_cluster):
    sos[c] = np.sum(np.square(corVal[:,c]))
    
r_sos = np.zeros(num_cluster)
for c in range(num_cluster):
    r_sos[c] = np.sum(np.square(r_corVal[:,c]))
In [110]:
plt.figure(figsize=(10,7))
plt.bar(np.linspace(-0.15,4.85,6),sos,width=0.4,color=sns.color_palette("Accent", n_colors=8)[4])
plt.bar(np.linspace(0.25,5.25,6),r_sos,width=0.4,color=sns.color_palette("Accent", n_colors=8)[5])
#plt.bar(np.linspace(-0.15,4.85,6),w/b,width=0.4,color=sns.color_palette("Accent", n_colors=8)[4])
#plt.bar(np.linspace(0.25,5.25,6),rw/rb,width=0.4,color=sns.color_palette("Accent", n_colors=8)[5])

plt.title('Sum of Squares of Correlations\nBetween Actions and Cluster',fontsize=25)
plt.xlabel('Cluster',fontsize=20)
plt.ylabel('Sum of Squares of Cluster Correlations',fontsize=20)
plt.legend(['Trained Agent','Random Agent'],fontsize=20)
plt.xticks(fontsize=15)
plt.yticks(fontsize=15)
#plt.show()
plt.savefig(path+'figures/Corrcoef_sum_squares.eps', bbox_inches='tight')
In [112]:
print('Overall sum of squares: Trained agent = '+str(np.round(np.sum(sos),3))+' Random Agent = '+str(np.round(np.sum(r_sos),3)))
Overall sum of squares: Trained agent = 0.413 Random Agent = 0.068
In [31]:
l_doors = label_test==6
l_doors = l_doors.astype(int)

fig = plt.figure(figsize=(15,5))
plt.subplots_adjust(top=0.83,wspace=0.01)
plt.subplot(1,2,1)
corrs = np.zeros((21,num_cluster))
for c in range(num_cluster):
    for r in range(-10,11):
        a = np.roll(l_doors, r)
        corrs[r+10,c] = np.corrcoef((cluster==c).astype(int),a)[1][0]
for c in range(num_cluster):
    r = num_cluster-c
    plt.plot(corrs[:,c],linewidth=5,color=(0.9, 0.4+r*0.1, 0.1 + r*0.07))
#plt.legend(np.linspace(0,num_cluster-1,num_cluster,dtype=int),title='Cluster',title_fontsize=15,fontsize=13,loc=2,ncol=2)
plt.title('Trained Agent',fontsize=18)
plt.xlabel('Sliding Window Offset',fontsize=15)
plt.ylabel('Correlation Coefficient',fontsize=15)
plt.xticks([0,5,10,15,20],['-10','-5','0','5','10    '],fontsize=13)
plt.ylim([-1,1])
plt.xlim([0,20])

plt.subplot(1,2,2)
r_corrs = np.zeros((21,num_cluster))
for c in range(num_cluster):
    for r in range(-10,11):
        a = np.roll(l_doors, r)
        r_corrs[r+10,c] = np.corrcoef((r_cluster==c).astype(int),a)[1][0]
for c in range(num_cluster):
    r = num_cluster-c
    plt.plot(r_corrs[:,c],linewidth=4,color=(0.9, 0.4+r*0.1, 0.1 + r*0.07))
plt.legend(np.linspace(0,num_cluster-1,num_cluster,dtype=int),title='Cluster',title_fontsize=15,fontsize=13,loc=2,ncol=2)
plt.title('Random Agent',fontsize=18)
plt.xlabel('Sliding Window Offset',fontsize=15)
#plt.ylabel('Correlation Coefficient',fontsize=15)
plt.xticks([0,5,10,15,20],['    -10','-5','0','5','10'],fontsize=13)
plt.yticks([],[])
plt.ylim([-1,1])
plt.xlim([0,20])

plt.suptitle('Correlation with Level Doors in the Visual Input',fontsize=20)
#plt.legend(fontsize=15,bbox_to_anchor=(1.15, 0.8), loc=2)
plt.show()
#plt.savefig(path+'figures/Corrcoef_Comparison_Doors.eps', bbox_inches='tight')
In [40]:
l_doors = label_test==6
l_doors = l_doors.astype(int)

fig = plt.figure(figsize=(10,7))
corrs = np.zeros((21,num_cluster))
for c in range(num_cluster):
    for r in range(-10,11):
        a = np.roll(l_doors, r)
        corrs[r+10,c] = np.corrcoef((cluster==c).astype(int),a)[1][0]
for c in range(num_cluster):
    r = num_cluster-c
    plt.plot(corrs[:,c],linewidth=5,color=(0.9, 0.4+r*0.1, 0.1 + r*0.07))
#plt.legend(np.linspace(0,num_cluster-1,num_cluster,dtype=int),title='Cluster',title_fontsize=15,fontsize=13,loc=2,ncol=2)
plt.title('Trained Agent',fontsize=18)
plt.xlabel('Sliding Window Offset',fontsize=15)
plt.ylabel('Correlation Coefficient',fontsize=15)
plt.legend(np.linspace(0,num_cluster-1,num_cluster,dtype=int),title='Cluster',title_fontsize=15,fontsize=15,loc=2,ncol=2)
plt.xticks([0,5,10,15,20],['-10','-5','0','5','10    '],fontsize=13)
plt.ylim([-0.2,0.6])
plt.xlim([0,20])

plt.suptitle('Correlation with Level Doors in the Visual Input',fontsize=20)
#plt.legend(fontsize=15,bbox_to_anchor=(1.15, 0.8), loc=2)
#plt.show()
#plt.savefig(path+'figures/Corrcoef_Comparison_Doors.eps', bbox_inches='tight')
plt.savefig(path+'figures/Corrcoef_Doors_Trained.png', bbox_inches='tight')
In [328]:
print("Trained Agent:\nMaximum Correlation: "+str(np.round(np.max(corrs),3))+" Minimum Correlation: "+str(np.round(np.min(corrs),3)))
print("Random Agent:\nMaximum Correlation: "+str(np.round(np.max(r_corrs),3))+" Minimum Correlation: "+str(np.round(np.min(r_corrs),3)))
Trained Agent:
Maximum Correlation: 0.421 Minimum Correlation: -0.113
Random Agent:
Maximum Correlation: 0.091 Minimum Correlation: -0.065

T-SNE (Conceptual Landscape, Generalization, Robustness)

In [8]:
from sklearn.manifold import TSNE
In [9]:
tsne = TSNE(n_components=2,perplexity=70,random_state=0).fit_transform(enc2D[:,:256])
#r_tsne = TSNE(n_components=2,perplexity=70,random_state=0).fit_transform(R_enc2D[:,:256])
In [676]:
door_cut = doors
door_cut[door_cut>0.5]=0

plt.figure(figsize=(10,7))
plt.title('T-SNE on the Visual Embedding\nTrained Agent', fontsize=20)
plt.scatter(tsne[:,0],tsne[:,1],c=door_cut,picker=True)
cbar = plt.colorbar(ticks = [0,0.5])
cbar.ax.set_yticklabels(['0%','>50%'],fontsize=15)
cbar.set_label('% of Pixels Showing Doors', rotation=270,fontsize=15)
plt.axis('off')
plt.show()
#plt.savefig(path+'figures/tsne_doors.eps', bbox_inches='tight')
In [117]:
import matplotlib as mpl
top = plt.cm.get_cmap('Dark2')
bottom = plt.cm.get_cmap('Blues')

newcolors = np.vstack((top(np.linspace(0, 1, 8)),
                       bottom(np.linspace(0.5, 1, 1))))
cmapnew = mpl.colors.ListedColormap(newcolors, name='GreyBlue')

sem_lab = ['Nothing','Entry Door','Normal Door',
           'Key Door','Puzzle Door','Level Door','Key','Orb','Puzzle']

plt.figure(figsize=(10,7))
plt.title('T-SNE on the Visual Embedding\nTrained Agent', fontsize=20)
im1 = plt.scatter(tsne[:,0],tsne[:,1],c=label_test,cmap=cmapnew)
values = np.linspace(1,9,9)
colors = [ im1.cmap(im1.norm(value)) for value in values]
patches = [ mpatches.Patch(color=colors[i], label=sem_lab[i] ) for i in range(len(values)) ]
plt.legend(handles=patches, bbox_to_anchor=(0.95, 0.8), loc=2, borderaxespad=0. , title='Image Content',title_fontsize=15,fontsize=13)

plt.axis('off')
#plt.show()
plt.savefig(path+'figures/tsne_semantic.eps', bbox_inches='tight')
In [54]:
label_test = np.array(hand_l['Label'])
edgeWs = np.array(hand_l['Label'])
edgeWs[edgeWs == 0]=0
edgeWs[edgeWs == 1]=0
edgeWs[edgeWs == 2]=0
edgeWs[edgeWs == 3]=0
edgeWs[edgeWs == 4]=0
edgeWs[edgeWs == 5]=0
edgeWs[edgeWs == 6]=1
edgeWs[edgeWs == 7]=0
edgeWs[edgeWs == 8]=0
edgeWs[edgeWs == 9]=0

edgeCs = np.array(hand_l['Label'])
edgeCs = edgeCs.astype(str)
edgeCs[edgeCs=='0'] = 'b'
edgeCs[edgeCs=='1'] = 'b'
edgeCs[edgeCs=='2'] = 'b'
edgeCs[edgeCs=='3'] = 'green'
edgeCs[edgeCs=='4'] = 'firebrick'
edgeCs[edgeCs=='5'] = 'purple'
edgeCs[edgeCs=='6'] = 'r'
edgeCs[edgeCs=='7'] = 'yellow'
edgeCs[edgeCs=='8'] = 'darkblue'
edgeCs[edgeCs=='9'] = 'b'
In [86]:
newOrder = np.append(np.where(edgeWs==0),np.where(edgeWs!=0))#to plot door frames on top (better visibility)
edgeWs[newOrder]

act_lab = ['Forward, Turn Right','Forward, Camera Left, Turn Right','Forward, Camera Left, Turn Left',
           'Forward, Camera Right, Turn Right','Backward, Camera Right, Turn Right','Everything Else']
fig = plt.figure(figsize=(10,7))
plt.title('T-SNE on the Visual Embedding\nTrained Agent', fontsize=20)
im1 = plt.scatter(tsne[newOrder,0],tsne[newOrder,1],c=all_a_comb[newOrder],cmap='Accent',edgecolors=edgeCs[newOrder],linewidths =edgeWs[newOrder])
values = np.linspace(0,5,6)
colors = [ im1.cmap(im1.norm(value)) for value in values]
patches = [ mpatches.Patch(color=colors[i], label=act_lab[i] ) for i in range(len(values)) ]
l1 = plt.legend(handles=patches, bbox_to_anchor=(0.91, 0.95), loc=2, borderaxespad=0. , title='Action Combination',title_fontsize=15,fontsize=13,frameon=False)

line1 = plt.Line2D(range(1), range(1), color="white", marker='o',markersize=7,markerfacecolor="white",markeredgecolor="red")
line2 = plt.Line2D(range(1), range(1), color="white", marker='o',markersize=7,markerfacecolor="white",markeredgecolor="white")
#line3 = plt.Line2D(range(1), range(1), color="white", marker='o',markersize=7,markerfacecolor="white",markeredgecolor="firebrick")
#line4 = plt.Line2D(range(1), range(1), color="white", marker='o',markersize=7,markerfacecolor="white",markeredgecolor="yellow")
#line5 = plt.Line2D(range(1), range(1), color="white", marker='o',markersize=7,markerfacecolor="white",markeredgecolor="darkblue")
l2 = plt.legend((line1,line2),('Level Door',''), bbox_to_anchor=(1, 0.55),numpoints=1, loc=2,fontsize=13,title="Observation Content",title_fontsize=15,frameon=False, borderaxespad=0.)

fig.add_artist(l1)
fig.add_artist(l2)

plt.axis('off')
plt.show()
#plt.savefig(path+'figures/tsne_actions_LevelDoors.eps', bbox_inches='tight')
#plt.savefig(path+'figures/tsne_actions_LevelDoors.png', bbox_inches='tight')
In [685]:
newOrder = np.append(np.where(edgeWs==0),np.where(edgeWs!=0))#to plot door frames on top (better visibility)
edgeWs[newOrder]

act_lab = ['Forward, Turn Right','Forward, Camera Left, Turn Right','Forward, Camera Left, Turn Left',
           'Forward, Camera Right, Turn Right','Backward, Camera Right, Turn Right','Everything Else']
fig = plt.figure(figsize=(10,7))
plt.title('T-SNE on the Visual Embedding\nRandom Agent', fontsize=20)
im1 = plt.scatter(r_tsne[newOrder,0],r_tsne[newOrder,1],c=all_a_comb[newOrder],cmap='Accent',edgecolors=edgeCs[newOrder],linewidths =edgeWs[newOrder])
values = np.linspace(0,5,6)
colors = [ im1.cmap(im1.norm(value)) for value in values]
patches = [ mpatches.Patch(color=colors[i], label=act_lab[i] ) for i in range(len(values)) ]
l1 = plt.legend(handles=patches, bbox_to_anchor=(0.91, 0.95), loc=2, borderaxespad=0. , title='Action Combination',title_fontsize=15,fontsize=13,frameon=False)

line1 = plt.Line2D(range(1), range(1), color="white", marker='o',markersize=7,markerfacecolor="white",markeredgecolor="red")
line2 = plt.Line2D(range(1), range(1), color="white", marker='o',markersize=7,markerfacecolor="white",markeredgecolor="white")
#line3 = plt.Line2D(range(1), range(1), color="white", marker='o',markersize=7,markerfacecolor="white",markeredgecolor="firebrick")
#line4 = plt.Line2D(range(1), range(1), color="white", marker='o',markersize=7,markerfacecolor="white",markeredgecolor="yellow")
#line5 = plt.Line2D(range(1), range(1), color="white", marker='o',markersize=7,markerfacecolor="white",markeredgecolor="darkblue")
l2 = plt.legend((line1,line2),('Level Door',''), bbox_to_anchor=(1, 0.55),numpoints=1, loc=2,fontsize=13,title="Observation Content",title_fontsize=15,frameon=False, borderaxespad=0.)

fig.add_artist(l1)
fig.add_artist(l2)

plt.axis('off')
plt.show()
#plt.savefig(path+'figures/r_tsne_actions_LevelDoors.eps', bbox_inches='tight')
In [759]:
label_test = np.array(hand_l['Label'])
edgeWs = np.array(hand_l['Label'])
edgeWs[edgeWs == 0]=0
edgeWs[edgeWs == 1]=0
edgeWs[edgeWs == 2]=0
edgeWs[edgeWs == 3]=1
edgeWs[edgeWs == 4]=1
edgeWs[edgeWs == 5]=1
edgeWs[edgeWs == 6]=1
edgeWs[edgeWs == 7]=1
edgeWs[edgeWs == 8]=1
edgeWs[edgeWs == 9]=0

newOrder = np.append(np.where(edgeWs==0),np.where(edgeWs!=0))#to plot door frames on top (better visibility)
edgeWs[newOrder]

act_lab = ['Forward, Turn Right','Forward, Camera Left, Turn Right','Forward, Camera Left, Turn Left',
           'Forward, Camera Right, Turn Right','Backward, Camera Right, Turn Right','Everything Else']
fig = plt.figure(figsize=(10,7))
plt.title('T-SNE on the Visual Embedding\nTrained Agent', fontsize=20)
im1 = plt.scatter(tsne[newOrder,0],tsne[newOrder,1],c=all_a_comb[newOrder],cmap='Accent',edgecolors=edgeCs[newOrder],linewidths =edgeWs[newOrder])
values = np.linspace(0,5,6)
colors = [ im1.cmap(im1.norm(value)) for value in values]
patches = [ mpatches.Patch(color=colors[i], label=act_lab[i] ) for i in range(len(values)) ]
l1 = plt.legend(handles=patches, bbox_to_anchor=(0.95, 0.95), loc=2, borderaxespad=0. , title='Action Combination',title_fontsize=15,fontsize=13)

line1 = plt.Line2D(range(1), range(1), color="white", marker='o',markersize=7,markerfacecolor="white",markeredgecolor="red")
line2 = plt.Line2D(range(1), range(1), color="white", marker='o',markersize=7,markerfacecolor="white",markeredgecolor="green")
line3 = plt.Line2D(range(1), range(1), color="white", marker='o',markersize=7,markerfacecolor="white",markeredgecolor="firebrick")
line4 = plt.Line2D(range(1), range(1), color="white", marker='o',markersize=7,markerfacecolor="white",markeredgecolor="yellow")
line5 = plt.Line2D(range(1), range(1), color="white", marker='o',markersize=7,markerfacecolor="white",markeredgecolor="darkblue")
l2 = plt.legend((line1,line2,line3,line4,line5),('Level Door','Normal Door', 'Key Door', 'Key','Orb'), bbox_to_anchor=(0.99, 0.55),numpoints=1, loc=2,fontsize=13,title="Observation Content",title_fontsize=15)

fig.add_artist(l1)
fig.add_artist(l2)

plt.axis('off')
plt.show()
#plt.savefig(path+'figures/tsne_actions_rewContent.eps', bbox_inches='tight')
In [689]:
plt.figure(figsize=(10,7))
plt.title('T-SNE on the Visual Embedding\nRandom Agent', fontsize=20)
plt.scatter(r_tsne[:,0],r_tsne[:,1],c=doors,picker=True)
cbar = plt.colorbar(ticks = [0,0.99])
cbar.ax.set_yticklabels(['0%','100%'],fontsize=15)
cbar.set_label('% of Pixels Showing Doors', rotation=270,fontsize=15)
plt.axis('off')
plt.show()
#plt.savefig(path+'figures/tsne_doors_r.eps', bbox_inches='tight')
In [331]:
from mpl_toolkits.mplot3d import Axes3D
tsne3 = TSNE(n_components=3,perplexity=30).fit_transform(enc2D[:,:256])
#r_tsne3 = TSNE(n_components=3,perplexity=30).fit_transform(R_enc2D[:,:256])
In [334]:
fig = plt.figure(figsize=(10,7))
ax = Axes3D(fig)
s = ax.scatter(tsne3[:,0],tsne3[:,1],tsne3[:,2],c=all_a_comb)
plt.title('T-SNE on the Visual Embedding',fontsize = 20)
#cbar = plt.colorbar(s,ticks = [0,0.99])
#cbar.ax.set_yticklabels(['0%','100%'],fontsize=15)
#cbar.set_label('% of Pixels Showing Doors', rotation=270,fontsize=15)
#plt.axis('off')
plt.show()
#plt.savefig(path+'figures/tsne_doors3D.eps', bbox_inches='tight')

Interactive TSNE

In [87]:
import pandas as pd
import mpld3
from mpld3 import plugins

css = """"""

fig, ax = plt.subplots(figsize=(10,10))
ax.grid(True, alpha=0.3)

N = 4000

#XX change this path for website
labels = ['<html><body><img title="{}" src="./frames/obs/img{}.png" width="100" height="100" alt="{}"></body></html>'.format(str(i),str(i),str(i)) for i in range(N)]

for p in newOrder:
    points2 = ax.plot(tsne[p,0],tsne[p,1],'o',color=colors[int(all_a_comb[p])],mec=edgeCs[p], ms=7, mew=edgeWs[p])


points = ax.plot(tsne[:,0],tsne[:,1],'o',color='slateblue',mec='navy', ms=7, mew=1,alpha=0.0)

ax.set_xlabel('')
ax.set_ylabel('')
ax.set_title('TSNE on the Visual Embedding', size=20)

tooltip = plugins.PointHTMLTooltip(points[0], labels,
                                   voffset=10, hoffset=10, css=css)
plugins.connect(fig, tooltip)
mpld3.save_html(fig,path+'tsne.html')
#mpld3.display()
In [86]:
%%capture
#only do this once!
for i in range(3998):
    plt.figure()
    plt.imshow(obs[i])
    plt.axis('off')
    plt.title(str(i),fontsize=25)
    plt.savefig(path+'frames/obs/img'+str(i)+'.png', bbox_inches='tight')
    #plt.show()

Cluster Variance in High-D Space

In [225]:
num_act = np.unique(all_a_comb).shape[0]

ba,wa = getClusterVariance(all_a_comb,enc2D[:,:256],num_act)
b = np.mean(ba)
w = np.zeros(num_act)
for i in range(num_act):
    w[i] = np.mean(wa[i])

rba,rwa = getClusterVariance(all_a_comb,R_enc2D[:,:256],num_act)
rb = np.mean(rba)
rw = np.zeros(num_act)
for i in range(num_act):
    rw[i] = np.mean(rwa[i])
In [226]:
print("Trained Agent: Between class variance: "+str(np.round(b,4)))
print("Trained Agent: within class variance:"+str(np.round(w,4)))
print("Relative class variance: "+str(np.round(w/b,4)))
print("Average class variance: "+str(np.round(np.mean(w/b),4)))
print("Average class variance, class 0-4: "+str(np.round(np.mean(w[:4]/b),4))+" Last 2: "+str(np.round(np.mean(w[4:]/b),4)))

print("Random Agent: Between class variance: "+str(np.round(rb,4)))
print("Trained Agent: within class variance:"+str(np.round(rw,4)))
print("Relative class variance: "+str(np.round(rw/rb,4)))
print("Average class variance: "+str(np.round(np.mean(rw/rb),4)))
print("Average class variance, class 0-4: "+str(np.round(np.mean(rw[:4]/rb),4))+" Last 2: "+str(np.round(np.mean(rw[4:]/rb),4)))
Trained Agent: Between class variance: 23.0799
Trained Agent: within class variance:[16.6139 13.4569 14.7458  8.9584 32.4644 44.1316]
Relative class variance: [0.7198 0.5831 0.6389 0.3881 1.4066 1.9121]
Average class variance: 0.9414
Average class variance, class 0-4: 0.5825 Last 2: 1.6594
Random Agent: Between class variance: 1e-04
Trained Agent: within class variance:[0.0001 0.0001 0.0002 0.0001 0.0001 0.0002]
Relative class variance: [0.8943 1.1039 1.3715 0.886  1.0537 1.5518]
Average class variance: 1.1435
Average class variance, class 0-4: 1.0639 Last 2: 1.3027
In [235]:
action_C = ['Forward,\nTurn Right','Forward,\nCamera Left,\nTurn Right','Forward,\nCamera Left,\nTurn Left',
           'Forward,\nCamera Right,\nTurn Right','Backward,\nCamera Right,\nTurn Right','Everything\nElse']

plt.figure(figsize=(10,7))
plt.bar(np.linspace(-0.15,4.85,6),w/b,width=0.4,color=sns.color_palette("Accent", n_colors=8)[4])
plt.bar(np.linspace(0.25,5.25,6),rw/rb,width=0.4,color=sns.color_palette("Accent", n_colors=8)[5])

plt.title('Variance within Encodings Corresponding to one Action',fontsize=20)
plt.xlabel('Action',fontsize=15)
plt.ylabel('Action Variance/Overall Variance',fontsize=15)
plt.legend(['Trained Agent','Random Agent'],fontsize=15)

plt.yticks(fontsize=15)
plt.xticks(np.linspace(0,5,6),action_C,fontsize=10)
plt.savefig(path+'figures/HighD_Action_Variances.eps', bbox_inches='tight')
#plt.show()

Cluster Point Distances in High-D Space

In [236]:
from itertools import combinations
from scipy.spatial import distance
In [238]:
def getDistances(points):
    distances = [distance.euclidean(p1, p2) for p1, p2 in combinations(points, 2)]
    return distances

num_act = np.unique(all_a_comb).shape[0]

enc2D_vis = enc2D[:,:256]
R_enc2D_vis = R_enc2D[:,:256]

allD = getDistances(enc2D_vis)
allD_var = np.var(allD)
allD = np.mean(allD)

acc_d = np.zeros(num_act)
acc_d_var = np.zeros(num_act)
for acc in range(num_act):
    acc_d[acc] = np.mean(getDistances(enc2D_vis[all_a_comb==acc]))
    acc_d_var[acc] = np.var(getDistances(enc2D_vis[all_a_comb==acc]))

r_allD = getDistances(R_enc2D_vis)
r_allD_var = np.var(r_allD)
r_allD = np.mean(r_allD)

r_acc_d = np.zeros(num_act)
r_acc_d_var = np.zeros(num_act)
for acc in range(num_act):
    r_acc_d[acc] = np.mean(getDistances(R_enc2D_vis[all_a_comb==acc]))
    r_acc_d_var[acc] = np.var(getDistances(R_enc2D_vis[all_a_comb==acc]))
In [340]:
print("Trained Agent:")
print("Average distance within between action clusters/overall distances: "+str(np.round(np.mean(acc_d)/allD,3)))
print("For first four actions: "+str(np.round(np.mean(acc_d[:4])/allD,3)))
print("Last two actions: "+str(np.round(np.mean(acc_d[4:])/allD,3)))
Trained Agent:
Average distance within between action clusters/overall distances: 0.96
For first four actions: 0.78
Last two actions: 1.319
In [341]:
print("Random Agent:")
print("Average distance within between action clusters/overall distances: "+str(np.round(np.mean(r_acc_d)/r_allD,3)))
print("For first four actions: "+str(np.round(np.mean(r_acc_d[:4])/r_allD,3)))
print("Last two actions: "+str(np.round(np.mean(r_acc_d[4:])/r_allD,3)))
Random Agent:
Average distance within between action clusters/overall distances: 1.063
For first four actions: 1.028
Last two actions: 1.133
In [242]:
action_C = ['Forward,\nTurn Right','Forward,\nCamera Left,\nTurn Right','Forward,\nCamera Left,\nTurn Left',
           'Forward,\nCamera Right,\nTurn Right','Backward,\nCamera Right,\nTurn Right','Everything\nElse']

plt.figure(figsize=(10,7))
plt.bar(np.linspace(-0.15,4.85,6),acc_d/allD,width=0.4,color=sns.color_palette("Accent", n_colors=8)[4])
plt.bar(np.linspace(0.25,5.25,6),r_acc_d/r_allD,width=0.4,color=sns.color_palette("Accent", n_colors=8)[5])

plt.title('Distance Within Encodings Corresponding to one Action',fontsize=20)
plt.xlabel('Action',fontsize=15)
plt.ylabel('Action Distance/Overall Distance',fontsize=15)
plt.legend(['Trained Agent','Random Agent'],fontsize=15)

plt.yticks(fontsize=15)
plt.xticks(np.linspace(0,5,6),action_C,fontsize=10)
plt.savefig(path+'figures/HighD_Action_Distances.eps', bbox_inches='tight')
#plt.show()

Doors

In [343]:
label_test = np.array(hand_l['Label'])
doorFrames = label_test
doorFrames[doorFrames!=6] = 0
doorFrames[doorFrames==6] = 1
In [344]:
plt.hist(doorFrames)
Out[344]:
(array([3817.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,
         183.]),
 array([0. , 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. ]),
 <a list of 10 Patch objects>)
In [345]:
doorD = getDistances(enc2D_vis[doorFrames==1])
noDoorD = getDistances(enc2D_vis[doorFrames==0])

r_doorD = getDistances(R_enc2D_vis[doorFrames==1])
r_noDoorD = getDistances(R_enc2D_vis[doorFrames==0])
In [348]:
print("Trained Agent")
print("Door frames: Average Distance = "+str(np.round(np.mean(doorD)/allD,3))+" Var = "+str(np.round(np.var(doorD),3)))
print("No Door frames: Average Distance = "+str(np.round(np.mean(noDoorD)/allD,3))+" Var = "+str(np.round(np.var(noDoorD),3)))

print("Random Agent")
print("Door frames: Average Distance = "+str(np.round(np.mean(r_doorD)/r_allD,3))+" Var = "+str(np.round(np.var(r_doorD),3)))
print("No Door frames: Average Distance = "+str(np.round(np.mean(r_noDoorD)/r_allD,3))+" Var = "+str(np.round(np.var(r_noDoorD),3)))
Trained Agent
Door frames: Average Distance = 2.025 Var = 9135.037
No Door frames: Average Distance = 0.923 Var = 1835.446
Random Agent
Door frames: Average Distance = 0.949 Var = 0.002
No Door frames: Average Distance = 1.001 Var = 0.003
In [349]:
doorM = np.mean(doorD)
In [351]:
nodoorM = np.mean(noDoorD)
In [352]:
print(allD)
print(doorM)
print(nodoorM)
95.22096001706906
192.83631543844575
87.90752853723373

Appendix - Run Once, Preprocessing

Extract Semantic Labels from RGB and get Percentages

In [ ]:
def getRewardPer(semantic):
    #Returns percentage of pixels containing a rewarding object -> door, key, orb, exit, doorZone, lockedDoorRegular
    rewarding = [2,3,4,7,8]
    r = 0
    for i in range(semantic.shape[0]):
        if semantic[i] in rewarding:
            r = r + 1
    return r/semantic.shape[0]
def getDoorPer(semantic):
    #Returns percentage of pixels containing a rewarding object -> door, key, orb, exit, doorZone, lockedDoorRegular
    rewarding = [2,3,4]#+8 for door area
    r = 0
    for i in range(semantic.shape[0]):
        if semantic[i] in rewarding:
            r = r + 1
    return r/semantic.shape[0]

labels = np.zeros(obs.shape)
for im_path in glob.glob(path+"LabelsRGB/*.png"):
    im = imageio.imread(im_path)
    labels[int(im_path.split("img")[1].split('.')[0])] = im
    
rewardS = []
doors = []
for i in range(len(labels)):
    l = rgb2L(labels[i]).flatten()
    np.save(path+'Labels/label'+str(i)+'.npy',l)
    rewardS = np.append(rewardS,getRewardPer(l))
    doors = np.append(doors,getDoorPer(l))
    if i%100==0:
        print(i)